import requests
import re
import os
import base64
from selenium import webdriver
from time import sleep

if __name__ == "__main__":
    # 创建一个文件夹，保存所有的图片
    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_argument('--headless')
    chrome_options.add_argument('--disable-gpu')

    # 1.创建Chrome浏览器对象，这会在电脑上在打开一个浏览器窗口
    browser = webdriver.Chrome(chrome_options=chrome_options, executable_path="D:\program\chromedriver.exe")
    url = 'https://www.ca8a2c7f0faf.com/tupian/list-%E8%87%AA%E6%8B%8D%E5%81%B7%E6%8B%8D.html'
    url_pre='https://www.ca8a2c7f0faf.com/tupian/list-%E8%87%AA%E6%8B%8D%E5%81%B7%E6%8B%8D'
    url_end='.html'
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36'
    }

    all_data = requests.get(url=url,headers=headers).text

    #print(all_data)
    #获取最大页码,聚焦页码标签
    page='<a class="hidden-xs" href=".*?>(.*?)</a>'
    page_nums = re.findall(page,all_data,re.S)
    pages_len=len(page_nums)
    pages_max_num= page_nums[pages_len-1]
    print("一共有"+pages_max_num+"页")
    for pp in range(int(pages_max_num)):
        dir_name = "./"

        dir_name =dir_name+"第"+(pp+1).__str__()+"页"
        print("==========开始第"+(pp+1).__str__()+"页")
        current_url=""
        if pp==0:
            current_url=url_pre+url_end
        else:
            current_url=url_pre+'-'+pp.__str__()+url_end

        print(current_url)
        all_data_2 = requests.get(url=current_url,headers=headers).text

        page_pre='https://www.ca8a2c7f0faf.com/'
        page='<li>.*?<a href="(.*?)" title.*?</li>'
        image_classes = re.findall(page,all_data_2,re.S)
        for cc in image_classes:
            dir_name=dir_name+cc+"组图"
            print("==========开始解析第" + (pp + 1).__str__() + "页"+"====第"+cc+"组")
            images_url=page_pre+cc
            print(images_url)


            print("浏览器加载完成!")

            browser.implicitly_wait(2)
            # 2.通过浏览器向服务器发送URL请求
            browser.get(images_url)

            # sleep(15)
            # 滚动到最下面
            # browser.execute_script("window.scrollTo(0,document.body.scrollHeight);")
            # sleep(15)
            images = browser.find_elements_by_class_name('videopic')
            couts = len(images)
            print(couts)
            if not os.path.exists(dir_name):
                os.makedirs(dir_name)
            for image in images:
                print(image.get_attribute("src"))
                # browser.execute_script("window.scrollTo(0,document.body.scrollHeight);")
                # 滑动滚动条到某个指定的元素
                js4 = "arguments[0].scrollIntoView();"
                # 将下拉滑动条滑动到当前div区域
                browser.execute_script(js4, image)

            # 3.刷新浏览器
            # browser.refresh()
            sleep(5)
            print("=================================")
            images_2 = browser.find_elements_by_class_name('videopic')
            image_num=0
            for image in images_2:
                image_num=image_num+1
                #print(image.get_attribute("src"))
                strtmp=image.get_attribute("src")
                missing_padding = 4 - len(strtmp) % 4
                if missing_padding:
                    strtmp += '=' * missing_padding
                else:
                    tmp = base64.b64decode(strtmp)
                    file_name = dir_name +"/"+ image_num.__str__() + '.txt'
                    # with open(file_name, 'wb') as f:
                    #     f.write(tmp)

            # all_data_3=requests.get(url=images_url,headers=headers).text
            # #print(all_data_3)
            # page_2 = '<img class="videopic lazy" src="(.*?)" data.*?>'
            # images = re.findall(page_2, all_data_3, re.S)
            # for image in images:
            #     print(image)
            #print(images_url)


    #
    # # 使用聚焦爬虫将所有的图片进行爬取,指定标签class
    # ex = '<div class="thumb">.*?<img src="(.*?)" alt.*?</div>'
    # img_src_list = re.findall(ex,all_data,re.S)
    # print(img_src_list)
    # for src in img_src_list:
    #     src = 'https:'+src
    #     img_data = requests.get(url=src,headers=headers).content
    #     # 生成图片名称
    #     img_name = src.split('/')[-1]
    #     # 图片存储路径
    #     imgPath = './糗图百科/' + img_name
    #     with open(imgPath,'wb') as fp:
    #         fp.write(img_data)
    #     print(img_name + '下载成功！！！')
